{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# `match_weights_chart`\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "tags": [
          "hide_input"
        ]
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "<style>\n",
              "  #altair-viz-5535b2542a56418db1e694b0e786c772.vega-embed {\n",
              "    width: 100%;\n",
              "    display: flex;\n",
              "  }\n",
              "\n",
              "  #altair-viz-5535b2542a56418db1e694b0e786c772.vega-embed details,\n",
              "  #altair-viz-5535b2542a56418db1e694b0e786c772.vega-embed details summary {\n",
              "    position: relative;\n",
              "  }\n",
              "</style>\n",
              "<div id=\"altair-viz-5535b2542a56418db1e694b0e786c772\"></div>\n",
              "<script type=\"text/javascript\">\n",
              "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
              "  (function(spec, embedOpt){\n",
              "    let outputDiv = document.currentScript.previousElementSibling;\n",
              "    if (outputDiv.id !== \"altair-viz-5535b2542a56418db1e694b0e786c772\") {\n",
              "      outputDiv = document.getElementById(\"altair-viz-5535b2542a56418db1e694b0e786c772\");\n",
              "    }\n",
              "    const paths = {\n",
              "      \"vega\": \"https://cdn.jsdelivr.net/npm/vega@5?noext\",\n",
              "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm/vega-lib?noext\",\n",
              "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm/vega-lite@5.17.0?noext\",\n",
              "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm/vega-embed@6?noext\",\n",
              "    };\n",
              "\n",
              "    function maybeLoadScript(lib, version) {\n",
              "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
              "      return (VEGA_DEBUG[key] == version) ?\n",
              "        Promise.resolve(paths[lib]) :\n",
              "        new Promise(function(resolve, reject) {\n",
              "          var s = document.createElement('script');\n",
              "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
              "          s.async = true;\n",
              "          s.onload = () => {\n",
              "            VEGA_DEBUG[key] = version;\n",
              "            return resolve(paths[lib]);\n",
              "          };\n",
              "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
              "          s.src = paths[lib];\n",
              "        });\n",
              "    }\n",
              "\n",
              "    function showError(err) {\n",
              "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
              "      throw err;\n",
              "    }\n",
              "\n",
              "    function displayChart(vegaEmbed) {\n",
              "      vegaEmbed(outputDiv, spec, embedOpt)\n",
              "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
              "    }\n",
              "\n",
              "    if(typeof define === \"function\" && define.amd) {\n",
              "      requirejs.config({paths});\n",
              "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
              "    } else {\n",
              "      maybeLoadScript(\"vega\", \"5\")\n",
              "        .then(() => maybeLoadScript(\"vega-lite\", \"5.17.0\"))\n",
              "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
              "        .catch(showError)\n",
              "        .then(() => displayChart(vegaEmbed));\n",
              "    }\n",
              "  })({\"config\": {\"view\": {\"continuousWidth\": 300, \"continuousHeight\": 300, \"discreteHeight\": 60, \"discreteWidth\": 400}, \"header\": {\"title\": null}, \"mark\": {\"tooltip\": null}, \"title\": {\"anchor\": \"middle\"}}, \"vconcat\": [{\"mark\": {\"type\": \"bar\", \"clip\": true, \"height\": 15}, \"encoding\": {\"color\": {\"field\": \"log2_bayes_factor\", \"scale\": {\"domain\": [-10, 0, 10], \"interpolate\": \"lab\", \"range\": [\"red\", \"#bbbbbb\", \"green\"]}, \"title\": \"Match weight\", \"type\": \"quantitative\"}, \"tooltip\": [{\"field\": \"comparison_name\", \"title\": \"Comparison name\", \"type\": \"nominal\"}, {\"field\": \"probability_two_random_records_match\", \"format\": \".4f\", \"title\": \"Probability two random records match\", \"type\": \"nominal\"}, {\"field\": \"log2_bayes_factor\", \"format\": \",.4f\", \"title\": \"Equivalent match weight\", \"type\": \"quantitative\"}, {\"field\": \"bayes_factor_description\", \"title\": \"Match weight description\", \"type\": \"nominal\"}], \"x\": {\"axis\": {\"domain\": false, \"gridColor\": {\"condition\": {\"test\": \"abs(datum.value / 10)  <= 1 & datum.value % 10 === 0\", \"value\": \"#aaa\"}, \"value\": \"#ddd\"}, \"gridDash\": {\"condition\": {\"test\": \"abs(datum.value / 10) == 1\", \"value\": [3]}, \"value\": null}, \"gridWidth\": {\"condition\": {\"test\": \"abs(datum.value / 10)  <= 1 & datum.value % 10 === 0\", \"value\": 2}, \"value\": 1}, \"labels\": false, \"ticks\": false, \"title\": \"\"}, \"field\": \"log2_bayes_factor\", \"scale\": {\"domain\": [-14, 14]}, \"type\": \"quantitative\"}, \"y\": {\"axis\": {\"title\": \"Prior (starting) match weight\", \"titleAlign\": \"right\", \"titleAngle\": 0, \"titleFontWeight\": \"normal\"}, \"field\": \"label_for_charts\", \"sort\": {\"field\": \"comparison_vector_value\", \"order\": \"descending\"}, \"type\": \"nominal\"}}, \"height\": 20, \"transform\": [{\"filter\": \"(datum.comparison_name == 'probability_two_random_records_match')\"}]}, {\"mark\": {\"type\": \"bar\", \"clip\": true}, \"encoding\": {\"color\": {\"field\": \"log2_bayes_factor\", \"scale\": {\"domain\": [-10, 0, 10], \"interpolate\": \"lab\", \"range\": [\"red\", \"#bbbbbb\", \"green\"]}, \"title\": \"Match weight\", \"type\": \"quantitative\"}, \"row\": {\"field\": \"comparison_name\", \"header\": {\"labelAlign\": \"left\", \"labelAnchor\": \"middle\", \"labelAngle\": 0}, \"sort\": {\"field\": \"comparison_sort_order\"}, \"type\": \"nominal\"}, \"tooltip\": [{\"field\": \"comparison_name\", \"title\": \"Comparison name\", \"type\": \"nominal\"}, {\"field\": \"label_for_charts\", \"title\": \"Label\", \"type\": \"ordinal\"}, {\"field\": \"sql_condition\", \"title\": \"SQL condition\", \"type\": \"nominal\"}, {\"field\": \"m_probability\", \"format\": \".10~g\", \"title\": \"M probability\", \"type\": \"quantitative\"}, {\"field\": \"u_probability\", \"format\": \".10~g\", \"title\": \"U probability\", \"type\": \"quantitative\"}, {\"field\": \"bayes_factor\", \"format\": \",.6f\", \"title\": \"Bayes factor = m/u\", \"type\": \"quantitative\"}, {\"field\": \"log2_bayes_factor\", \"format\": \".4~g\", \"title\": \"Match weight = log2(m/u)\", \"type\": \"quantitative\"}, {\"field\": \"bayes_factor_description\", \"title\": \"Match weight description\", \"type\": \"nominal\"}, {\"field\": \"m_probability_description\", \"title\": \"m probability description\", \"type\": \"nominal\"}, {\"field\": \"u_probability_description\", \"title\": \"u probability description\", \"type\": \"nominal\"}], \"x\": {\"axis\": {\"gridColor\": {\"condition\": {\"test\": \"abs(datum.value / 10)  <= 1 & datum.value % 10 === 0\", \"value\": \"#aaa\"}, \"value\": \"#ddd\"}, \"gridDash\": {\"condition\": {\"test\": \"abs(datum.value / 10) == 1\", \"value\": [3]}, \"value\": null}, \"gridWidth\": {\"condition\": {\"test\": \"abs(datum.value / 10)  <= 1 & datum.value % 10 === 0\", \"value\": 2}, \"value\": 1}, \"title\": \"Comparison level match weight = log2(m/u)\"}, \"field\": \"log2_bayes_factor\", \"scale\": {\"domain\": [-14, 14]}, \"type\": \"quantitative\"}, \"y\": {\"axis\": {\"title\": null}, \"field\": \"label_for_charts\", \"sort\": {\"field\": \"comparison_vector_value\", \"order\": \"descending\"}, \"type\": \"nominal\"}}, \"height\": {\"step\": 12}, \"resolve\": {\"axis\": {\"y\": \"independent\"}, \"scale\": {\"y\": \"independent\"}}, \"transform\": [{\"filter\": \"(datum.comparison_name != 'probability_two_random_records_match')\"}]}], \"data\": {\"name\": \"data-f8036c696494227b15612de7bd982b35\"}, \"params\": [{\"name\": \"mouse_zoom\", \"select\": {\"type\": \"interval\", \"encodings\": [\"x\"]}, \"bind\": \"scales\", \"views\": []}], \"resolve\": {\"axis\": {\"y\": \"independent\"}, \"scale\": {\"y\": \"independent\"}}, \"title\": {\"text\": \"Model parameters (components of final match weight)\", \"subtitle\": \"Use mousewheel to zoom\"}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.9.3.json\", \"datasets\": {\"data-f8036c696494227b15612de7bd982b35\": [{\"comparison_name\": \"probability_two_random_records_match\", \"sql_condition\": null, \"label_for_charts\": \"\", \"m_probability\": null, \"u_probability\": null, \"m_probability_description\": null, \"u_probability_description\": null, \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": null, \"is_null_level\": false, \"bayes_factor\": 0.00010001000100010001, \"log2_bayes_factor\": -13.287568102831404, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 0, \"bayes_factor_description\": \"The probability that two random records drawn at random match is 0.000 or one in  10,000.0 records.This is equivalent to a starting match weight of -13.288.\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": -1}, {\"comparison_name\": \"first_name\", \"sql_condition\": \"\\\"first_name_l\\\" = \\\"first_name_r\\\"\", \"label_for_charts\": \"Exact match on first_name\", \"m_probability\": 0.49136441060423774, \"u_probability\": 0.0057935713975033705, \"m_probability_description\": \"Amongst matching record comparisons, 49.14% of records (i.e. one in 2.035) are in the exact match on first_name comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.5794% of records (i.e. one in 173) are in the exact match on first_name comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 84.81200573725248, \"log2_bayes_factor\": 6.406196597784454, \"comparison_vector_value\": 3, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on first_name` then comparison is 84.81 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 0}, {\"comparison_name\": \"first_name\", \"sql_condition\": \"jaro_winkler_similarity(\\\"first_name_l\\\", \\\"first_name_r\\\") >= 0.9\", \"label_for_charts\": \"Jaro-Winkler distance of first_name >= 0.9\", \"m_probability\": 0.1914735181842002, \"u_probability\": 0.003386832528670639, \"m_probability_description\": \"Amongst matching record comparisons, 19.15% of records (i.e. one in 5.223) are in the jaro-winkler distance of first_name >= 0.9 comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.3387% of records (i.e. one in 295) are in the jaro-winkler distance of first_name >= 0.9 comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 56.53468737037175, \"log2_bayes_factor\": 5.821064412712825, \"comparison_vector_value\": 2, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `jaro-winkler distance of first_name >= 0.9` then comparison is 56.53 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 0}, {\"comparison_name\": \"first_name\", \"sql_condition\": \"jaro_winkler_similarity(\\\"first_name_l\\\", \\\"first_name_r\\\") >= 0.7\", \"label_for_charts\": \"Jaro-Winkler distance of first_name >= 0.7\", \"m_probability\": 0.11346321825427946, \"u_probability\": 0.019439490815246544, \"m_probability_description\": \"Amongst matching record comparisons, 11.35% of records (i.e. one in 8.813) are in the jaro-winkler distance of first_name >= 0.7 comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 1.944% of records (i.e. one in 51.44) are in the jaro-winkler distance of first_name >= 0.7 comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 5.83673818067752, \"log2_bayes_factor\": 2.5451623545127915, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `jaro-winkler distance of first_name >= 0.7` then comparison is 5.837 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 0}, {\"comparison_name\": \"first_name\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.20369885295728277, \"u_probability\": 0.9713801052585794, \"m_probability_description\": \"Amongst matching record comparisons, 20.37% of records (i.e. one in 4.909) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 97.14% of records (i.e. one in 1.029) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.2097004579922486, \"log2_bayes_factor\": -2.2535980825590833, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 4.769 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 0}, {\"comparison_name\": \"surname\", \"sql_condition\": \"\\\"surname_l\\\" = \\\"surname_r\\\"\", \"label_for_charts\": \"Exact match on surname\", \"m_probability\": 0.4345244833248351, \"u_probability\": 0.004889975550122249, \"m_probability_description\": \"Amongst matching record comparisons, 43.45% of records (i.e. one in 2.301) are in the exact match on surname comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.489% of records (i.e. one in 204) are in the exact match on surname comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 88.86025683992878, \"log2_bayes_factor\": 6.473466406178517, \"comparison_vector_value\": 3, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on surname` then comparison is 88.86 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 1}, {\"comparison_name\": \"surname\", \"sql_condition\": \"jaro_similarity(\\\"surname_l\\\", \\\"surname_r\\\") >= 0.9\", \"label_for_charts\": \"Jaro distance of 'surname >= 0.9'\", \"m_probability\": 0.21637408339607053, \"u_probability\": 0.0025524597651737017, \"m_probability_description\": \"Amongst matching record comparisons, 21.64% of records (i.e. one in 4.622) are in the jaro distance of 'surname >= 0.9' comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.2552% of records (i.e. one in 392) are in the jaro distance of 'surname >= 0.9' comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 84.77081063071947, \"log2_bayes_factor\": 6.405495677982012, \"comparison_vector_value\": 2, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `jaro distance of 'surname >= 0.9'` then comparison is 84.77 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 1}, {\"comparison_name\": \"surname\", \"sql_condition\": \"jaro_similarity(\\\"surname_l\\\", \\\"surname_r\\\") >= 0.7\", \"label_for_charts\": \"Jaro distance of 'surname >= 0.7'\", \"m_probability\": 0.13100004234324156, \"u_probability\": 0.01614766651441468, \"m_probability_description\": \"Amongst matching record comparisons, 13.1% of records (i.e. one in 7.634) are in the jaro distance of 'surname >= 0.7' comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 1.615% of records (i.e. one in 61.93) are in the jaro distance of 'surname >= 0.7' comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 8.112629910105005, \"log2_bayes_factor\": 3.0201696756280945, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `jaro distance of 'surname >= 0.7'` then comparison is 8.113 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 1}, {\"comparison_name\": \"surname\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.2181013909358528, \"u_probability\": 0.9764098981702893, \"m_probability_description\": \"Amongst matching record comparisons, 21.81% of records (i.e. one in 4.585) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 97.64% of records (i.e. one in 1.024) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.22337072918305784, \"log2_bayes_factor\": -2.162487949483276, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 4.477 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 1}, {\"comparison_name\": \"dob\", \"sql_condition\": \"\\\"dob_l\\\" = \\\"dob_r\\\"\", \"label_for_charts\": \"Exact match on date of birth\", \"m_probability\": 0.3899839392762551, \"u_probability\": 0.0017477477477477479, \"m_probability_description\": \"Amongst matching record comparisons, 39% of records (i.e. one in 2.564) are in the exact match on date of birth comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.1748% of records (i.e. one in 572) are in the exact match on date of birth comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 223.13514051373357, \"log2_bayes_factor\": 7.801773924569989, \"comparison_vector_value\": 4, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `exact match on date of birth` then comparison is 223 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"dob\", \"sql_condition\": \"damerau_levenshtein(\\\"dob_l\\\", \\\"dob_r\\\") <= 1\", \"label_for_charts\": \"DamerauLevenshtein distance <= 1\", \"m_probability\": 0.14884650150455297, \"u_probability\": 0.0016436436436436436, \"m_probability_description\": \"Amongst matching record comparisons, 14.88% of records (i.e. one in 6.718) are in the dameraulevenshtein distance <= 1 comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.1644% of records (i.e. one in 608) are in the dameraulevenshtein distance <= 1 comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 90.55886419186872, \"log2_bayes_factor\": 6.500783958589023, \"comparison_vector_value\": 3, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `dameraulevenshtein distance <= 1` then comparison is 90.56 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"dob\", \"sql_condition\": \"ABS(EPOCH(try_strptime(\\\"dob_l\\\", '%Y-%m-%d')) - EPOCH(try_strptime(\\\"dob_r\\\", '%Y-%m-%d'))) <= 31557600.0\", \"label_for_charts\": \"Abs date difference <= 1 year\", \"m_probability\": 0.19399707416347942, \"u_probability\": 0.03546146146146146, \"m_probability_description\": \"Amongst matching record comparisons, 19.4% of records (i.e. one in 5.155) are in the abs date difference <= 1 year comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 3.546% of records (i.e. one in 28.2) are in the abs date difference <= 1 year comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 5.470645206608591, \"log2_bayes_factor\": 2.4517109941661124, \"comparison_vector_value\": 2, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `abs date difference <= 1 year` then comparison is 5.471 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"dob\", \"sql_condition\": \"ABS(EPOCH(try_strptime(\\\"dob_l\\\", '%Y-%m-%d')) - EPOCH(try_strptime(\\\"dob_r\\\", '%Y-%m-%d'))) <= 2629800.0\", \"label_for_charts\": \"Abs date difference <= 1 month\", \"m_probability\": 0.012500000000000011, \"u_probability\": 0.06299605249474372, \"m_probability_description\": \"Amongst matching record comparisons, 1.25% of records (i.e. one in 80) are in the abs date difference <= 1 month comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 6.3% of records (i.e. one in 15.87) are in the abs date difference <= 1 month comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.19842513149602492, \"log2_bayes_factor\": -2.3333333333333335, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `abs date difference <= 1 month` then comparison is 5.04 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"dob\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.2671724850557125, \"u_probability\": 0.9611471471471471, \"m_probability_description\": \"Amongst matching record comparisons, 26.72% of records (i.e. one in 3.743) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 96.11% of records (i.e. one in 1.04) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.2779725100872714, \"log2_bayes_factor\": -1.8469858792830283, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 3.597 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"city\", \"sql_condition\": \"\\\"city_l\\\" = \\\"city_r\\\"\", \"label_for_charts\": \"Exact match on city\", \"m_probability\": 0.5611221596314009, \"u_probability\": 0.0551475711801453, \"m_probability_description\": \"Amongst matching record comparisons, 56.11% of records (i.e. one in 1.782) are in the exact match on city comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 5.515% of records (i.e. one in 18.13) are in the exact match on city comparison level\", \"has_tf_adjustments\": true, \"tf_adjustment_column\": \"city\", \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 10.174920628842143, \"log2_bayes_factor\": 3.3469456354405605, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 1, \"bayes_factor_description\": \"If comparison level is `exact match on city` then comparison is 10.17 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 3}, {\"comparison_name\": \"city\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.4388778403685992, \"u_probability\": 0.9448524288198547, \"m_probability_description\": \"Amongst matching record comparisons, 43.89% of records (i.e. one in 2.279) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 94.49% of records (i.e. one in 1.058) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.4644935303989948, \"log2_bayes_factor\": -1.1062695924156825, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 1, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 2.153 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 3}, {\"comparison_name\": \"email\", \"sql_condition\": \"\\\"email_l\\\" = \\\"email_r\\\"\", \"label_for_charts\": \"Exact match on email\", \"m_probability\": 0.552153409103165, \"u_probability\": 0.0021938713143283602, \"m_probability_description\": \"Amongst matching record comparisons, 55.22% of records (i.e. one in 1.811) are in the exact match on email comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.2194% of records (i.e. one in 456) are in the exact match on email comparison level\", \"has_tf_adjustments\": true, \"tf_adjustment_column\": \"email\", \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 251.67994380390687, \"log2_bayes_factor\": 7.975446443510344, \"comparison_vector_value\": 4, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `exact match on email` then comparison is 252 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}, {\"comparison_name\": \"email\", \"sql_condition\": \"NULLIF(regexp_extract(\\\"email_l\\\", '^[^@]+', 0), '') = NULLIF(regexp_extract(\\\"email_r\\\", '^[^@]+', 0), '')\", \"label_for_charts\": \"Exact match on username\", \"m_probability\": 0.22055262276218593, \"u_probability\": 0.0010390328952024346, \"m_probability_description\": \"Amongst matching record comparisons, 22.06% of records (i.e. one in 4.534) are in the exact match on username comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.1039% of records (i.e. one in 962) are in the exact match on username comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 212.26721866126837, \"log2_bayes_factor\": 7.729737776625641, \"comparison_vector_value\": 3, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `exact match on username` then comparison is 212 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}, {\"comparison_name\": \"email\", \"sql_condition\": \"jaro_winkler_similarity(\\\"email_l\\\", \\\"email_r\\\") >= 0.88\", \"label_for_charts\": \"Jaro-Winkler distance of email >= 0.88\", \"m_probability\": 0.21383894038442586, \"u_probability\": 0.0009135769109519858, \"m_probability_description\": \"Amongst matching record comparisons, 21.38% of records (i.e. one in 4.676) are in the jaro-winkler distance of email >= 0.88 comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.09136% of records (i.e. one in 1,095) are in the jaro-winkler distance of email >= 0.88 comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 234.06780296318635, \"log2_bayes_factor\": 7.870782688940884, \"comparison_vector_value\": 2, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `jaro-winkler distance of email >= 0.88` then comparison is 234 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}, {\"comparison_name\": \"email\", \"sql_condition\": \"jaro_winkler_similarity(NULLIF(regexp_extract(\\\"email_l\\\", '^[^@]+', 0), ''), NULLIF(regexp_extract(\\\"email_r\\\", '^[^@]+', 0), '')) >= 0.88\", \"label_for_charts\": \"Jaro-Winkler >0.88 on username\", \"m_probability\": 0.012500000000000011, \"u_probability\": 0.000501823937001795, \"m_probability_description\": \"Amongst matching record comparisons, 1.25% of records (i.e. one in 80) are in the jaro-winkler >0.88 on username comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.05018% of records (i.e. one in 1,993) are in the jaro-winkler >0.88 on username comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 24.909134615384634, \"log2_bayes_factor\": 4.638602995720225, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `jaro-winkler >0.88 on username` then comparison is 24.91 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}, {\"comparison_name\": \"email\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.013455027750223171, \"u_probability\": 0.9953516949425154, \"m_probability_description\": \"Amongst matching record comparisons, 1.346% of records (i.e. one in 74.32) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 99.54% of records (i.e. one in 1.005) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.01351786290071093, \"log2_bayes_factor\": -6.208989102226405, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 73.98 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}]}}, {\"mode\": \"vega-lite\"});\n",
              "</script>"
            ],
            "text/plain": [
              "alt.VConcatChart(...)"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "chart"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n",
        "!!! info \"At a glance\"\n",
        "    **Useful for:** Looking at the whole Splink model definition.\n",
        "\n",
        "    **API Documentation:** [match_weights_chart()](../api_docs/visualisations.md#splink.internals.linker_components.visualisations.LinkerVisualisations.match_weights_chart)\n",
        "\n",
        "    **What is needed to generate the chart?** A trained Splink model."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### What the chart shows\n",
        "\n",
        "The `match_weights_chart` show the results of a trained Splink model. Each comparison within a model is represented in a bar chart, with a bar showing the evidence for two records being a match (i.e. match weight) for each comparison level.\n",
        "\n",
        "??? note \"What the chart tooltip shows\"\n",
        "\n",
        "    ![](./img/match_weights_chart_tooltip.png)\n",
        "\n",
        "    The tooltip shows information based on the comparison level bar that the user is hovering over, including:\n",
        "\n",
        "    - The name of the comparison and comaprison level.\n",
        "    - The comparison level condition as an SQL statement.\n",
        "    - The m and u proability for the comparison level.\n",
        "    - The resulting bayes factor and match weight for the comparison level."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### How to interpret the chart\n",
        "\n",
        "Each bar in the `match_weights_chart` shows the evidence of a match provided by each level in a Splink model (i.e. match weight). As such, the match weight chart provides a summary for the entire Splink model, as it shows the match weights for every type of comparison defined within the model.\n",
        "\n",
        "Any Splink score generated to compare two records will add up the evidence (i.e. match weights) for each comparison to come up with a final match weight score, which can then be converted into a probability of a match.\n",
        "\n",
        "The first bar chart is the Prior Match Weight, which is the . This can be thought of in the same way as the y-intercept of a simple regression model\n",
        "\n",
        "This chart is an aggregation of the [`m_u_parameters_chart`](./m_u_parameters_chart.ipynb). The match weight for a comparison level is simply $log_2(\\frac{m}{u})$."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Actions to take as a result of the chart\n",
        "\n",
        "Some heuristics to help assess Splink models with the `match_weights_chart`:\n",
        "\n",
        "#### Match weights gradually reducing within a comparison\n",
        "\n",
        "Comparison levels are order dependent, therefore they are constructed that the most \"similar\" levels come first and get gradually less \"similar\". As a result, we would generally expect that match weight will reduce as we go down the levels in a comparison. \n",
        "\n",
        "#### Very similar comparison levels\n",
        "\n",
        "Comparisons are broken up into comparison levels to show different levels of similarity between records. As these levels are associated with different levels of similarity, we expect the amount of evidence (i.e. match weight) to vary between comparison levels. Two levels with the same match weight does not provide the model with any additional information which could make it perform better. \n",
        "\n",
        "Therefore, if two levels of a comparison return the same match weight, these should be combined into a single level.\n",
        "\n",
        "#### Very different comparison levels\n",
        "\n",
        "Levels that have a large variation between comparison levels have a significant impact on the model results. For example, looking at the `email` comparison in the chart above, the difference in match weight between an exact/fuzzy match and \"All other comparisons\" is > 13, which is quite extreme. This generally happens with highly predictive features (e.g. email, national insurance number, social security number).\n",
        "\n",
        "If there are a number of highly predictive features, it is worth looking at simplifying your model using these more predictive features. In some cases, similar results may be obtained with a [deterministic](../topic_guides/theory/probabilistic_vs_deterministic.md) rather than a probabilistic linkage model.\n",
        "\n",
        "#### Logical Walk-through\n",
        "\n",
        "One of the most effective methods to assess a splink model is to walk through each of the comparison levels of the `match_weights_chart` and sense check the amount of evidence (i.e. match weight) that has been allocated by the model.\n",
        "\n",
        "For example, in the chart above, we would expect records with the same `dob` to provide more evidence of a match that `first_name` or `surname`. Conversely, given how people can move location, we would expect that `city` would be less predictive than people's fixed, personally identifying characteristics like `surname`, `dob` etc.\n",
        "\n",
        "#### Anything look strange?\n",
        "\n",
        "If anything still looks unusual, check out:\n",
        "\n",
        "- the underlying m and u values in the [`m_u_parameters_chart`](./m_u_parameters_chart.ipynb)\n",
        "- the values from each training session in the [`parameter_estimate_comparisons_chart`](./parameter_estimate_comparisons_chart.ipynb)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Related Charts\n",
        "\n",
        "::cards::\n",
        "[\n",
        "    {\n",
        "    \"title\": \"`m u parameters chart`\",\n",
        "    \"image\": \"./img/m_u_parameters_chart.png\",\n",
        "    \"url\": \"./m_u_parameters_chart.ipynb\"\n",
        "    },\n",
        "    {\n",
        "    \"title\": \"`parameter estimate comparisons chart`\",\n",
        "    \"image\": \"./img/parameter_estimate_comparisons_chart.png\",\n",
        "    \"url\": \"./parameter_estimate_comparisons_chart.ipynb\"\n",
        "    },\n",
        "]\n",
        "::/cards::"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Worked Example"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "tags": [
          "hide_output"
        ]
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "You are using the default value for `max_pairs`, which may be too small and thus lead to inaccurate estimates for your model's u-parameters. Consider increasing to 1e8 or 1e9, which will result in more accurate estimates, but with a longer run time.\n",
            "----- Estimating u probabilities using random sampling -----\n",
            "u probability not trained for dob - Abs date difference <= 1 month (comparison vector value: 1). This usually means the comparison level was never observed in the training data.\n",
            "\n",
            "Estimated u probabilities using random sampling\n",
            "\n",
            "Your model is not yet fully trained. Missing estimates for:\n",
            "    - first_name (no m values are trained).\n",
            "    - surname (no m values are trained).\n",
            "    - dob (some u values are not trained, no m values are trained).\n",
            "    - city (no m values are trained).\n",
            "    - email (no m values are trained).\n",
            "\n",
            "----- Starting EM training session -----\n",
            "\n",
            "Estimating the m probabilities of the model by blocking on:\n",
            "(l.\"first_name\" = r.\"first_name\") AND (l.\"surname\" = r.\"surname\")\n",
            "\n",
            "Parameter estimates will be made for the following comparison(s):\n",
            "    - dob\n",
            "    - city\n",
            "    - email\n",
            "\n",
            "Parameter estimates cannot be made for the following comparison(s) since they are used in the blocking rules: \n",
            "    - first_name\n",
            "    - surname\n",
            "\n",
            "WARNING:\n",
            "Level Abs date difference <= 1 month on comparison dob not observed in dataset, unable to train m value\n",
            "\n",
            "WARNING:\n",
            "Level Jaro-Winkler >0.88 on username on comparison email not observed in dataset, unable to train m value\n",
            "\n",
            "Iteration 1: Largest change in params was -0.463 in the m_probability of dob, level `Exact match on date of birth`\n",
            "Iteration 2: Largest change in params was 0.144 in probability_two_random_records_match\n",
            "Iteration 3: Largest change in params was 0.0328 in probability_two_random_records_match\n",
            "Iteration 4: Largest change in params was 0.0108 in probability_two_random_records_match\n",
            "Iteration 5: Largest change in params was 0.00444 in probability_two_random_records_match\n",
            "Iteration 6: Largest change in params was 0.00212 in probability_two_random_records_match\n",
            "Iteration 7: Largest change in params was 0.00111 in probability_two_random_records_match\n",
            "Iteration 8: Largest change in params was 0.000611 in probability_two_random_records_match\n",
            "Iteration 9: Largest change in params was 0.000347 in probability_two_random_records_match\n",
            "Iteration 10: Largest change in params was 0.0002 in probability_two_random_records_match\n",
            "Iteration 11: Largest change in params was 0.000117 in probability_two_random_records_match\n",
            "Iteration 12: Largest change in params was 6.85e-05 in probability_two_random_records_match\n",
            "\n",
            "EM converged after 12 iterations\n",
            "m probability not trained for dob - Abs date difference <= 1 month (comparison vector value: 1). This usually means the comparison level was never observed in the training data.\n",
            "m probability not trained for email - Jaro-Winkler >0.88 on username (comparison vector value: 1). This usually means the comparison level was never observed in the training data.\n",
            "\n",
            "Your model is not yet fully trained. Missing estimates for:\n",
            "    - first_name (no m values are trained).\n",
            "    - surname (no m values are trained).\n",
            "    - dob (some u values are not trained, some m values are not trained).\n",
            "    - email (some m values are not trained).\n",
            "\n",
            "----- Starting EM training session -----\n",
            "\n",
            "Estimating the m probabilities of the model by blocking on:\n",
            "l.\"dob\" = r.\"dob\"\n",
            "\n",
            "Parameter estimates will be made for the following comparison(s):\n",
            "    - first_name\n",
            "    - surname\n",
            "    - city\n",
            "    - email\n",
            "\n",
            "Parameter estimates cannot be made for the following comparison(s) since they are used in the blocking rules: \n",
            "    - dob\n",
            "\n",
            "WARNING:\n",
            "Level Jaro-Winkler >0.88 on username on comparison email not observed in dataset, unable to train m value\n",
            "\n",
            "Iteration 1: Largest change in params was 0.632 in probability_two_random_records_match\n",
            "Iteration 2: Largest change in params was 0.173 in probability_two_random_records_match\n",
            "Iteration 3: Largest change in params was 0.0865 in the m_probability of first_name, level `All other comparisons`\n",
            "Iteration 4: Largest change in params was 0.0354 in the m_probability of first_name, level `All other comparisons`\n",
            "Iteration 5: Largest change in params was 0.013 in probability_two_random_records_match\n",
            "Iteration 6: Largest change in params was 0.00552 in probability_two_random_records_match\n",
            "Iteration 7: Largest change in params was 0.00253 in probability_two_random_records_match\n",
            "Iteration 8: Largest change in params was 0.0012 in probability_two_random_records_match\n",
            "Iteration 9: Largest change in params was 0.000584 in probability_two_random_records_match\n",
            "Iteration 10: Largest change in params was 0.000286 in probability_two_random_records_match\n",
            "Iteration 11: Largest change in params was 0.000141 in probability_two_random_records_match\n",
            "Iteration 12: Largest change in params was 6.93e-05 in probability_two_random_records_match\n",
            "\n",
            "EM converged after 12 iterations\n",
            "m probability not trained for email - Jaro-Winkler >0.88 on username (comparison vector value: 1). This usually means the comparison level was never observed in the training data.\n",
            "\n",
            "Your model is not yet fully trained. Missing estimates for:\n",
            "    - dob (some u values are not trained, some m values are not trained).\n",
            "    - email (some m values are not trained).\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "<style>\n",
              "  #altair-viz-10da1603ec4e4eeb8b9984f81f88392d.vega-embed {\n",
              "    width: 100%;\n",
              "    display: flex;\n",
              "  }\n",
              "\n",
              "  #altair-viz-10da1603ec4e4eeb8b9984f81f88392d.vega-embed details,\n",
              "  #altair-viz-10da1603ec4e4eeb8b9984f81f88392d.vega-embed details summary {\n",
              "    position: relative;\n",
              "  }\n",
              "</style>\n",
              "<div id=\"altair-viz-10da1603ec4e4eeb8b9984f81f88392d\"></div>\n",
              "<script type=\"text/javascript\">\n",
              "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
              "  (function(spec, embedOpt){\n",
              "    let outputDiv = document.currentScript.previousElementSibling;\n",
              "    if (outputDiv.id !== \"altair-viz-10da1603ec4e4eeb8b9984f81f88392d\") {\n",
              "      outputDiv = document.getElementById(\"altair-viz-10da1603ec4e4eeb8b9984f81f88392d\");\n",
              "    }\n",
              "    const paths = {\n",
              "      \"vega\": \"https://cdn.jsdelivr.net/npm/vega@5?noext\",\n",
              "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm/vega-lib?noext\",\n",
              "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm/vega-lite@5.17.0?noext\",\n",
              "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm/vega-embed@6?noext\",\n",
              "    };\n",
              "\n",
              "    function maybeLoadScript(lib, version) {\n",
              "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
              "      return (VEGA_DEBUG[key] == version) ?\n",
              "        Promise.resolve(paths[lib]) :\n",
              "        new Promise(function(resolve, reject) {\n",
              "          var s = document.createElement('script');\n",
              "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
              "          s.async = true;\n",
              "          s.onload = () => {\n",
              "            VEGA_DEBUG[key] = version;\n",
              "            return resolve(paths[lib]);\n",
              "          };\n",
              "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
              "          s.src = paths[lib];\n",
              "        });\n",
              "    }\n",
              "\n",
              "    function showError(err) {\n",
              "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
              "      throw err;\n",
              "    }\n",
              "\n",
              "    function displayChart(vegaEmbed) {\n",
              "      vegaEmbed(outputDiv, spec, embedOpt)\n",
              "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
              "    }\n",
              "\n",
              "    if(typeof define === \"function\" && define.amd) {\n",
              "      requirejs.config({paths});\n",
              "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
              "    } else {\n",
              "      maybeLoadScript(\"vega\", \"5\")\n",
              "        .then(() => maybeLoadScript(\"vega-lite\", \"5.17.0\"))\n",
              "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
              "        .catch(showError)\n",
              "        .then(() => displayChart(vegaEmbed));\n",
              "    }\n",
              "  })({\"config\": {\"view\": {\"continuousWidth\": 300, \"continuousHeight\": 300, \"discreteHeight\": 60, \"discreteWidth\": 400}, \"header\": {\"title\": null}, \"mark\": {\"tooltip\": null}, \"title\": {\"anchor\": \"middle\"}}, \"vconcat\": [{\"mark\": {\"type\": \"bar\", \"clip\": true, \"height\": 15}, \"encoding\": {\"color\": {\"field\": \"log2_bayes_factor\", \"scale\": {\"domain\": [-10, 0, 10], \"interpolate\": \"lab\", \"range\": [\"red\", \"#bbbbbb\", \"green\"]}, \"title\": \"Match weight\", \"type\": \"quantitative\"}, \"tooltip\": [{\"field\": \"comparison_name\", \"title\": \"Comparison name\", \"type\": \"nominal\"}, {\"field\": \"probability_two_random_records_match\", \"format\": \".4f\", \"title\": \"Probability two random records match\", \"type\": \"nominal\"}, {\"field\": \"log2_bayes_factor\", \"format\": \",.4f\", \"title\": \"Equivalent match weight\", \"type\": \"quantitative\"}, {\"field\": \"bayes_factor_description\", \"title\": \"Match weight description\", \"type\": \"nominal\"}], \"x\": {\"axis\": {\"domain\": false, \"gridColor\": {\"condition\": {\"test\": \"abs(datum.value / 10)  <= 1 & datum.value % 10 === 0\", \"value\": \"#aaa\"}, \"value\": \"#ddd\"}, \"gridDash\": {\"condition\": {\"test\": \"abs(datum.value / 10) == 1\", \"value\": [3]}, \"value\": null}, \"gridWidth\": {\"condition\": {\"test\": \"abs(datum.value / 10)  <= 1 & datum.value % 10 === 0\", \"value\": 2}, \"value\": 1}, \"labels\": false, \"ticks\": false, \"title\": \"\"}, \"field\": \"log2_bayes_factor\", \"scale\": {\"domain\": [-14, 14]}, \"type\": \"quantitative\"}, \"y\": {\"axis\": {\"title\": \"Prior (starting) match weight\", \"titleAlign\": \"right\", \"titleAngle\": 0, \"titleFontWeight\": \"normal\"}, \"field\": \"label_for_charts\", \"sort\": {\"field\": \"comparison_vector_value\", \"order\": \"descending\"}, \"type\": \"nominal\"}}, \"height\": 20, \"transform\": [{\"filter\": \"(datum.comparison_name == 'probability_two_random_records_match')\"}]}, {\"mark\": {\"type\": \"bar\", \"clip\": true}, \"encoding\": {\"color\": {\"field\": \"log2_bayes_factor\", \"scale\": {\"domain\": [-10, 0, 10], \"interpolate\": \"lab\", \"range\": [\"red\", \"#bbbbbb\", \"green\"]}, \"title\": \"Match weight\", \"type\": \"quantitative\"}, \"row\": {\"field\": \"comparison_name\", \"header\": {\"labelAlign\": \"left\", \"labelAnchor\": \"middle\", \"labelAngle\": 0}, \"sort\": {\"field\": \"comparison_sort_order\"}, \"type\": \"nominal\"}, \"tooltip\": [{\"field\": \"comparison_name\", \"title\": \"Comparison name\", \"type\": \"nominal\"}, {\"field\": \"label_for_charts\", \"title\": \"Label\", \"type\": \"ordinal\"}, {\"field\": \"sql_condition\", \"title\": \"SQL condition\", \"type\": \"nominal\"}, {\"field\": \"m_probability\", \"format\": \".10~g\", \"title\": \"M probability\", \"type\": \"quantitative\"}, {\"field\": \"u_probability\", \"format\": \".10~g\", \"title\": \"U probability\", \"type\": \"quantitative\"}, {\"field\": \"bayes_factor\", \"format\": \",.6f\", \"title\": \"Bayes factor = m/u\", \"type\": \"quantitative\"}, {\"field\": \"log2_bayes_factor\", \"format\": \".4~g\", \"title\": \"Match weight = log2(m/u)\", \"type\": \"quantitative\"}, {\"field\": \"bayes_factor_description\", \"title\": \"Match weight description\", \"type\": \"nominal\"}, {\"field\": \"m_probability_description\", \"title\": \"m probability description\", \"type\": \"nominal\"}, {\"field\": \"u_probability_description\", \"title\": \"u probability description\", \"type\": \"nominal\"}], \"x\": {\"axis\": {\"gridColor\": {\"condition\": {\"test\": \"abs(datum.value / 10)  <= 1 & datum.value % 10 === 0\", \"value\": \"#aaa\"}, \"value\": \"#ddd\"}, \"gridDash\": {\"condition\": {\"test\": \"abs(datum.value / 10) == 1\", \"value\": [3]}, \"value\": null}, \"gridWidth\": {\"condition\": {\"test\": \"abs(datum.value / 10)  <= 1 & datum.value % 10 === 0\", \"value\": 2}, \"value\": 1}, \"title\": \"Comparison level match weight = log2(m/u)\"}, \"field\": \"log2_bayes_factor\", \"scale\": {\"domain\": [-14, 14]}, \"type\": \"quantitative\"}, \"y\": {\"axis\": {\"title\": null}, \"field\": \"label_for_charts\", \"sort\": {\"field\": \"comparison_vector_value\", \"order\": \"descending\"}, \"type\": \"nominal\"}}, \"height\": {\"step\": 12}, \"resolve\": {\"axis\": {\"y\": \"independent\"}, \"scale\": {\"y\": \"independent\"}}, \"transform\": [{\"filter\": \"(datum.comparison_name != 'probability_two_random_records_match')\"}]}], \"data\": {\"name\": \"data-f8036c696494227b15612de7bd982b35\"}, \"params\": [{\"name\": \"mouse_zoom\", \"select\": {\"type\": \"interval\", \"encodings\": [\"x\"]}, \"bind\": \"scales\", \"views\": []}], \"resolve\": {\"axis\": {\"y\": \"independent\"}, \"scale\": {\"y\": \"independent\"}}, \"title\": {\"text\": \"Model parameters (components of final match weight)\", \"subtitle\": \"Use mousewheel to zoom\"}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.9.3.json\", \"datasets\": {\"data-f8036c696494227b15612de7bd982b35\": [{\"comparison_name\": \"probability_two_random_records_match\", \"sql_condition\": null, \"label_for_charts\": \"\", \"m_probability\": null, \"u_probability\": null, \"m_probability_description\": null, \"u_probability_description\": null, \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": null, \"is_null_level\": false, \"bayes_factor\": 0.00010001000100010001, \"log2_bayes_factor\": -13.287568102831404, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 0, \"bayes_factor_description\": \"The probability that two random records drawn at random match is 0.000 or one in  10,000.0 records.This is equivalent to a starting match weight of -13.288.\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": -1}, {\"comparison_name\": \"first_name\", \"sql_condition\": \"\\\"first_name_l\\\" = \\\"first_name_r\\\"\", \"label_for_charts\": \"Exact match on first_name\", \"m_probability\": 0.49136441060423774, \"u_probability\": 0.0057935713975033705, \"m_probability_description\": \"Amongst matching record comparisons, 49.14% of records (i.e. one in 2.035) are in the exact match on first_name comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.5794% of records (i.e. one in 173) are in the exact match on first_name comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 84.81200573725248, \"log2_bayes_factor\": 6.406196597784454, \"comparison_vector_value\": 3, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on first_name` then comparison is 84.81 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 0}, {\"comparison_name\": \"first_name\", \"sql_condition\": \"jaro_winkler_similarity(\\\"first_name_l\\\", \\\"first_name_r\\\") >= 0.9\", \"label_for_charts\": \"Jaro-Winkler distance of first_name >= 0.9\", \"m_probability\": 0.1914735181842002, \"u_probability\": 0.003386832528670639, \"m_probability_description\": \"Amongst matching record comparisons, 19.15% of records (i.e. one in 5.223) are in the jaro-winkler distance of first_name >= 0.9 comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.3387% of records (i.e. one in 295) are in the jaro-winkler distance of first_name >= 0.9 comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 56.53468737037175, \"log2_bayes_factor\": 5.821064412712825, \"comparison_vector_value\": 2, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `jaro-winkler distance of first_name >= 0.9` then comparison is 56.53 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 0}, {\"comparison_name\": \"first_name\", \"sql_condition\": \"jaro_winkler_similarity(\\\"first_name_l\\\", \\\"first_name_r\\\") >= 0.7\", \"label_for_charts\": \"Jaro-Winkler distance of first_name >= 0.7\", \"m_probability\": 0.11346321825427946, \"u_probability\": 0.019439490815246544, \"m_probability_description\": \"Amongst matching record comparisons, 11.35% of records (i.e. one in 8.813) are in the jaro-winkler distance of first_name >= 0.7 comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 1.944% of records (i.e. one in 51.44) are in the jaro-winkler distance of first_name >= 0.7 comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 5.83673818067752, \"log2_bayes_factor\": 2.5451623545127915, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `jaro-winkler distance of first_name >= 0.7` then comparison is 5.837 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 0}, {\"comparison_name\": \"first_name\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.20369885295728277, \"u_probability\": 0.9713801052585794, \"m_probability_description\": \"Amongst matching record comparisons, 20.37% of records (i.e. one in 4.909) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 97.14% of records (i.e. one in 1.029) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.2097004579922486, \"log2_bayes_factor\": -2.2535980825590833, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 4.769 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 0}, {\"comparison_name\": \"surname\", \"sql_condition\": \"\\\"surname_l\\\" = \\\"surname_r\\\"\", \"label_for_charts\": \"Exact match on surname\", \"m_probability\": 0.4345244833248351, \"u_probability\": 0.004889975550122249, \"m_probability_description\": \"Amongst matching record comparisons, 43.45% of records (i.e. one in 2.301) are in the exact match on surname comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.489% of records (i.e. one in 204) are in the exact match on surname comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 88.86025683992878, \"log2_bayes_factor\": 6.473466406178517, \"comparison_vector_value\": 3, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on surname` then comparison is 88.86 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 1}, {\"comparison_name\": \"surname\", \"sql_condition\": \"jaro_similarity(\\\"surname_l\\\", \\\"surname_r\\\") >= 0.9\", \"label_for_charts\": \"Jaro distance of 'surname >= 0.9'\", \"m_probability\": 0.21637408339607053, \"u_probability\": 0.0025524597651737017, \"m_probability_description\": \"Amongst matching record comparisons, 21.64% of records (i.e. one in 4.622) are in the jaro distance of 'surname >= 0.9' comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.2552% of records (i.e. one in 392) are in the jaro distance of 'surname >= 0.9' comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 84.77081063071947, \"log2_bayes_factor\": 6.405495677982012, \"comparison_vector_value\": 2, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `jaro distance of 'surname >= 0.9'` then comparison is 84.77 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 1}, {\"comparison_name\": \"surname\", \"sql_condition\": \"jaro_similarity(\\\"surname_l\\\", \\\"surname_r\\\") >= 0.7\", \"label_for_charts\": \"Jaro distance of 'surname >= 0.7'\", \"m_probability\": 0.13100004234324156, \"u_probability\": 0.01614766651441468, \"m_probability_description\": \"Amongst matching record comparisons, 13.1% of records (i.e. one in 7.634) are in the jaro distance of 'surname >= 0.7' comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 1.615% of records (i.e. one in 61.93) are in the jaro distance of 'surname >= 0.7' comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 8.112629910105005, \"log2_bayes_factor\": 3.0201696756280945, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `jaro distance of 'surname >= 0.7'` then comparison is 8.113 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 1}, {\"comparison_name\": \"surname\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.2181013909358528, \"u_probability\": 0.9764098981702893, \"m_probability_description\": \"Amongst matching record comparisons, 21.81% of records (i.e. one in 4.585) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 97.64% of records (i.e. one in 1.024) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.22337072918305784, \"log2_bayes_factor\": -2.162487949483276, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 4.477 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 1}, {\"comparison_name\": \"dob\", \"sql_condition\": \"\\\"dob_l\\\" = \\\"dob_r\\\"\", \"label_for_charts\": \"Exact match on date of birth\", \"m_probability\": 0.3899839392762551, \"u_probability\": 0.0017477477477477479, \"m_probability_description\": \"Amongst matching record comparisons, 39% of records (i.e. one in 2.564) are in the exact match on date of birth comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.1748% of records (i.e. one in 572) are in the exact match on date of birth comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 223.13514051373357, \"log2_bayes_factor\": 7.801773924569989, \"comparison_vector_value\": 4, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `exact match on date of birth` then comparison is 223 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"dob\", \"sql_condition\": \"damerau_levenshtein(\\\"dob_l\\\", \\\"dob_r\\\") <= 1\", \"label_for_charts\": \"DamerauLevenshtein distance <= 1\", \"m_probability\": 0.14884650150455297, \"u_probability\": 0.0016436436436436436, \"m_probability_description\": \"Amongst matching record comparisons, 14.88% of records (i.e. one in 6.718) are in the dameraulevenshtein distance <= 1 comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.1644% of records (i.e. one in 608) are in the dameraulevenshtein distance <= 1 comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 90.55886419186872, \"log2_bayes_factor\": 6.500783958589023, \"comparison_vector_value\": 3, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `dameraulevenshtein distance <= 1` then comparison is 90.56 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"dob\", \"sql_condition\": \"ABS(EPOCH(try_strptime(\\\"dob_l\\\", '%Y-%m-%d')) - EPOCH(try_strptime(\\\"dob_r\\\", '%Y-%m-%d'))) <= 31557600.0\", \"label_for_charts\": \"Abs date difference <= 1 year\", \"m_probability\": 0.19399707416347942, \"u_probability\": 0.03546146146146146, \"m_probability_description\": \"Amongst matching record comparisons, 19.4% of records (i.e. one in 5.155) are in the abs date difference <= 1 year comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 3.546% of records (i.e. one in 28.2) are in the abs date difference <= 1 year comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 5.470645206608591, \"log2_bayes_factor\": 2.4517109941661124, \"comparison_vector_value\": 2, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `abs date difference <= 1 year` then comparison is 5.471 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"dob\", \"sql_condition\": \"ABS(EPOCH(try_strptime(\\\"dob_l\\\", '%Y-%m-%d')) - EPOCH(try_strptime(\\\"dob_r\\\", '%Y-%m-%d'))) <= 2629800.0\", \"label_for_charts\": \"Abs date difference <= 1 month\", \"m_probability\": 0.012500000000000011, \"u_probability\": 0.06299605249474372, \"m_probability_description\": \"Amongst matching record comparisons, 1.25% of records (i.e. one in 80) are in the abs date difference <= 1 month comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 6.3% of records (i.e. one in 15.87) are in the abs date difference <= 1 month comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.19842513149602492, \"log2_bayes_factor\": -2.3333333333333335, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `abs date difference <= 1 month` then comparison is 5.04 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"dob\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.2671724850557125, \"u_probability\": 0.9611471471471471, \"m_probability_description\": \"Amongst matching record comparisons, 26.72% of records (i.e. one in 3.743) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 96.11% of records (i.e. one in 1.04) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.2779725100872714, \"log2_bayes_factor\": -1.8469858792830283, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 3.597 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"city\", \"sql_condition\": \"\\\"city_l\\\" = \\\"city_r\\\"\", \"label_for_charts\": \"Exact match on city\", \"m_probability\": 0.5611221596314009, \"u_probability\": 0.0551475711801453, \"m_probability_description\": \"Amongst matching record comparisons, 56.11% of records (i.e. one in 1.782) are in the exact match on city comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 5.515% of records (i.e. one in 18.13) are in the exact match on city comparison level\", \"has_tf_adjustments\": true, \"tf_adjustment_column\": \"city\", \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 10.174920628842143, \"log2_bayes_factor\": 3.3469456354405605, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 1, \"bayes_factor_description\": \"If comparison level is `exact match on city` then comparison is 10.17 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 3}, {\"comparison_name\": \"city\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.4388778403685992, \"u_probability\": 0.9448524288198547, \"m_probability_description\": \"Amongst matching record comparisons, 43.89% of records (i.e. one in 2.279) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 94.49% of records (i.e. one in 1.058) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.4644935303989948, \"log2_bayes_factor\": -1.1062695924156825, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 1, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 2.153 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 3}, {\"comparison_name\": \"email\", \"sql_condition\": \"\\\"email_l\\\" = \\\"email_r\\\"\", \"label_for_charts\": \"Exact match on email\", \"m_probability\": 0.552153409103165, \"u_probability\": 0.0021938713143283602, \"m_probability_description\": \"Amongst matching record comparisons, 55.22% of records (i.e. one in 1.811) are in the exact match on email comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.2194% of records (i.e. one in 456) are in the exact match on email comparison level\", \"has_tf_adjustments\": true, \"tf_adjustment_column\": \"email\", \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 251.67994380390687, \"log2_bayes_factor\": 7.975446443510344, \"comparison_vector_value\": 4, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `exact match on email` then comparison is 252 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}, {\"comparison_name\": \"email\", \"sql_condition\": \"NULLIF(regexp_extract(\\\"email_l\\\", '^[^@]+', 0), '') = NULLIF(regexp_extract(\\\"email_r\\\", '^[^@]+', 0), '')\", \"label_for_charts\": \"Exact match on username\", \"m_probability\": 0.22055262276218593, \"u_probability\": 0.0010390328952024346, \"m_probability_description\": \"Amongst matching record comparisons, 22.06% of records (i.e. one in 4.534) are in the exact match on username comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.1039% of records (i.e. one in 962) are in the exact match on username comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 212.26721866126837, \"log2_bayes_factor\": 7.729737776625641, \"comparison_vector_value\": 3, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `exact match on username` then comparison is 212 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}, {\"comparison_name\": \"email\", \"sql_condition\": \"jaro_winkler_similarity(\\\"email_l\\\", \\\"email_r\\\") >= 0.88\", \"label_for_charts\": \"Jaro-Winkler distance of email >= 0.88\", \"m_probability\": 0.21383894038442586, \"u_probability\": 0.0009135769109519858, \"m_probability_description\": \"Amongst matching record comparisons, 21.38% of records (i.e. one in 4.676) are in the jaro-winkler distance of email >= 0.88 comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.09136% of records (i.e. one in 1,095) are in the jaro-winkler distance of email >= 0.88 comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 234.06780296318635, \"log2_bayes_factor\": 7.870782688940884, \"comparison_vector_value\": 2, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `jaro-winkler distance of email >= 0.88` then comparison is 234 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}, {\"comparison_name\": \"email\", \"sql_condition\": \"jaro_winkler_similarity(NULLIF(regexp_extract(\\\"email_l\\\", '^[^@]+', 0), ''), NULLIF(regexp_extract(\\\"email_r\\\", '^[^@]+', 0), '')) >= 0.88\", \"label_for_charts\": \"Jaro-Winkler >0.88 on username\", \"m_probability\": 0.012500000000000011, \"u_probability\": 0.000501823937001795, \"m_probability_description\": \"Amongst matching record comparisons, 1.25% of records (i.e. one in 80) are in the jaro-winkler >0.88 on username comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.05018% of records (i.e. one in 1,993) are in the jaro-winkler >0.88 on username comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 24.909134615384634, \"log2_bayes_factor\": 4.638602995720225, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `jaro-winkler >0.88 on username` then comparison is 24.91 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}, {\"comparison_name\": \"email\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.013455027750223171, \"u_probability\": 0.9953516949425154, \"m_probability_description\": \"Amongst matching record comparisons, 1.346% of records (i.e. one in 74.32) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 99.54% of records (i.e. one in 1.005) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.01351786290071093, \"log2_bayes_factor\": -6.208989102226405, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 73.98 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}]}}, {\"mode\": \"vega-lite\"});\n",
              "</script>"
            ],
            "text/plain": [
              "alt.VConcatChart(...)"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import splink.comparison_library as cl\n",
        "\n",
        "from splink import DuckDBAPI, Linker, SettingsCreator, block_on, splink_datasets\n",
        "\n",
        "df = splink_datasets.fake_1000\n",
        "\n",
        "settings = SettingsCreator(\n",
        "    link_type=\"dedupe_only\",\n",
        "    comparisons=[\n",
        "        cl.JaroWinklerAtThresholds(\"first_name\", [0.9, 0.7]),\n",
        "        cl.JaroAtThresholds(\"surname\", [0.9, 0.7]),\n",
        "        cl.DateOfBirthComparison(\n",
        "            \"dob\",\n",
        "            input_is_string=True,\n",
        "            datetime_metrics=[\"year\", \"month\"],\n",
        "            datetime_thresholds=[1, 1],\n",
        "        ),\n",
        "        cl.ExactMatch(\"city\").configure(term_frequency_adjustments=True),\n",
        "        cl.EmailComparison(\"email\"),\n",
        "    ],\n",
        "    blocking_rules_to_generate_predictions=[\n",
        "        block_on(\"first_name\"),\n",
        "        block_on(\"surname\"),\n",
        "    ],\n",
        ")\n",
        "\n",
        "linker = Linker(df, settings, DuckDBAPI())\n",
        "linker.training.estimate_u_using_random_sampling(max_pairs=1e6)\n",
        "\n",
        "blocking_rule_for_training = block_on(\"first_name\", \"surname\")\n",
        "\n",
        "linker.training.estimate_parameters_using_expectation_maximisation(\n",
        "    blocking_rule_for_training\n",
        ")\n",
        "\n",
        "blocking_rule_for_training = block_on(\"dob\")\n",
        "linker.training.estimate_parameters_using_expectation_maximisation(\n",
        "    blocking_rule_for_training\n",
        ")\n",
        "\n",
        "chart = linker.visualisations.match_weights_chart()\n",
        "chart\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "base",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.8"
    },
    "orig_nbformat": 4
  },
  "nbformat": 4,
  "nbformat_minor": 2
}
